{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "import re\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clearstring(string):\n",
    "    string = re.sub('[^\\'\\\"A-Za-z0-9 ]+', '', string)\n",
    "    string = string.split(' ')\n",
    "    string = filter(None, string)\n",
    "    string = [y.strip() for y in string]\n",
    "    string = [y for y in string if len(y) > 3 and y.find('nbsp') < 0]\n",
    "    return ' '.join(string)\n",
    "\n",
    "def read_data(location):\n",
    "    list_folder = os.listdir(location)\n",
    "    label = list_folder\n",
    "    label.sort()\n",
    "    outer_string, outer_label = [], []\n",
    "    for i in range(len(list_folder)):\n",
    "        list_file = os.listdir('data/' + list_folder[i])\n",
    "        strings = []\n",
    "        for x in range(len(list_file)):\n",
    "            with open('data/' + list_folder[i] + '/' + list_file[x], 'r') as fopen:\n",
    "                strings += fopen.read().split('\\n')\n",
    "        strings = list(filter(None, strings))\n",
    "        for k in range(len(strings)):\n",
    "            strings[k] = clearstring(strings[k])\n",
    "        labels = [i] * len(strings)\n",
    "        outer_string += strings\n",
    "        outer_label += labels\n",
    "    \n",
    "    dataset = np.array([outer_string, outer_label])\n",
    "    dataset = dataset.T\n",
    "    np.random.shuffle(dataset)\n",
    "        \n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 'woked feelin pain mood been tarified caused cant stnd hater could this life being hated throught whole career dont dont care thing thoe finna break that feel about babi carroline caused heart soul feelin real',\n",
       "        '4'],\n",
       "       [ 'couldnt stop feeling threatened cards grandmother kept sending with cash inside them',\n",
       "        '1'],\n",
       "       ['feel chronically defeated satisfied', '4'],\n",
       "       [ 'apologize anyone feels offended these remarks certainly welcome your opinions',\n",
       "        '0'],\n",
       "       [ 'feel like every romantic movie there dorky best friend that desperately love with beautiful leading girl',\n",
       "        '3']],\n",
       "      dtype='<U477')"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset = read_data('data/')\n",
    "dataset[:5,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('dataset-emotion.p', 'wb') as fopen:\n",
    "    pickle.dump(dataset, fopen)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
